package git.soulbgm.otcwuxi;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;

/**
 * www.otcwuxi.com爬虫
 *
 * @author SoulBGM
 * @date 2024-09-02
 */
public class OtcwuxiApplication {
    private static final String BASE_URL = "https://www.otcwuxi.com";
    private static final String WRITER_PATH = "/home/cjxnm.txt";

    public static void main(String[] args) throws IOException {
        Document doc = getDocument(BASE_URL + "/chapter/13876662111");
        Element listElement = doc.getElementById("list");
        Elements et = listElement.select("a");
        Path path = Paths.get(WRITER_PATH);
        if (!Files.exists(path)) {
            Files.createFile(path);
        }
        System.out.println("总计 " + et.size() + " 章");
        for (Element e : et) {
            String title = e.attr("title");
            String href = e.attr("href");
            Files.write(path, (title + "\n").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND);
            Document childDoc = getDocument(BASE_URL + href);
            Element contentElement = childDoc.getElementById("content");
            String html = contentElement.html();
            String content = html.replaceAll("<br> ", "").replaceAll("<!--go--> ", "").replaceAll("\n\n", "\n");
            Files.write(path, ("\n" + content + "\n\n").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND);
            System.out.println(title + " 下载完毕！");
        }
    }

    private static Document getDocument(String url) throws IOException {
        Connection connect = Jsoup.connect(url);
        connect.header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
        Connection.Response rs = connect.execute();
        // 转换为Dom树
        return Jsoup.parse(rs.body());
    }

}
